package com.gylang.crawler.crawler.dy2018;

import com.gylang.crawler.crawler.EsLogDownloader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Spider;

/**
 * @author gylang
 * data 2021/5/28
 */
@Component
public class Dy2018PageListProcessor extends BaseDy2018PageProcessor{

    @Autowired
    private Dy2018CrawlerPageProcessor crawlerPageProcessor;
    @Autowired
    private EsLogDownloader esLogDownloader;
    @Override
    public void process(Page page) {
        // 下一页
        String[] url = page.getHtml().xpath("//*[@name='select']/option/@value").nodes().stream()
                .map(s -> BASE_URL + s.toString()).toArray(String[]::new);
        Spider.create(crawlerPageProcessor)
                .setDownloader(esLogDownloader)
                .addUrl(url)
                .thread(20)
                .runAsync();
    }
}
